#include <iostream>
#include <string>
#include <vector>
#include <boost/filesystem.hpp>
#include "util.hpp"

const std::string src_path = "data/input/";
const std::string output = "data/raw_html/raw.txt";

// 文档格式
typedef struct DocInfo
{
	std::string title;		
	std::string content;
	std::string url;
}DocInfo_t;

bool EnumFile(const std::string &src_path, std::vector<std::string> *files_list)
{
	namespace fs = boost::filesystem;
	fs::path root_path(src_path);
	if (fs::exists(root_path) == false) // 判断路径是否存在
	{
		std::cerr << src_path << " not exists" << std::endl;
		return false;
	}
	fs::recursive_directory_iterator end;
	for (fs::recursive_directory_iterator it(root_path); it != end; it++)
	{
		if (fs::is_regular_file(*it) == false) // 判断是否是普通文件
		{
			continue;
		}
		if (it->path().extension() != ".html") // 判断文件路径名是否符合要求
		{
			continue;
		}
		//std::cout << "debug: " << it->path().string() << std::endl;
		files_list->push_back(it->path().string());
	}
	return true;
}

static bool ParseTitle(const std::string &file, std::string *title)
{
	size_t begin = file.find("<title>");
	if (begin == std::string::npos) return false;
	size_t end = file.find("</title>");
	if (end == std::string::npos) return false;
	begin += std::string("<title>").size();
	if (begin > end) return false;
	*title = file.substr(begin, end - begin);
	return true;
}

static bool ParseContent(const std::string &file, std::string *content)
{
	// 状态机
	enum status
	{
		LABLE,
		CONTENT
	};

	enum status s = LABLE; // 开始默认为标签
	for (char c : file)
	{
		switch (s)
		{
		case LABLE : if (c == '>') s = CONTENT; break; // 如果遇到'>'，假设接下来是content
		case CONTENT : 
			if (c == '<') s = LABLE; // 如果假设错误，状态重新转为lable
			else
			{
				// 后面我们想用\n作为html解析后文本的分隔符
				if (c == '\n') c = ' ';
				content->push_back(c);
			}
			break;
		default : break;
		}
	}
	return true;
}

static bool ParseUrl(const std::string &file_path, std::string *url)
{
	std::string url_head = "https://www.boost.org/doc/libs/1_88_0/doc/html/";
	std::string url_tail = file_path.substr(src_path.size());
	*url = url_head + url_tail;
	return true;
}

bool ParseHtml(const std::vector<std::string> &files_list, std::vector<DocInfo_t> *results)
{
	for (const std::string &file : files_list)
	{
		std::string result;
		if (yjz_util::FileUtil::ReadFile(file, &result) == false) continue;
		DocInfo_t doc;
		if (ParseTitle(result, &doc.title) == false) continue;
		if (ParseContent(result, &doc.content) == false) continue;
		if (ParseUrl(file, &doc.url) == false) continue;
		results->push_back(std::move(doc));
	}
	return true;
}

bool SaveHtml(const std::vector<DocInfo_t> &results, const std::string &output)
{
#define SEP '\3'
	// 按二进制方式写入
	std::ofstream out(output, std::ios::out | std::ios::binary);
	if (out.is_open() == false)
	{
		std::cerr << "open " << output << " fail!" << std::endl;
		return false;
	}
	for (auto &it : results)
	{
		std::string out_string;
		out_string = it.title;
		out_string += SEP;
		out_string += it.content;
		out_string += SEP;
		out_string += it.url;
		out_string += '\n';
		out.write(out_string.c_str(), out_string.size());
	}
	out.close();
	return true;
}

int main()
{
	// 1.把每个html文件名和路径保存起来，方便后续文件读取
	std::vector<std::string> files_list;
	if (EnumFile(src_path, &files_list) == false)
	{
		std::cerr << "enum file name fail!" << std::endl;
		return 1;
	}

	// 2.按照files_list读取每个文件的内容，并进行解析
	std::vector<DocInfo_t> results;
	if (ParseHtml(files_list, &results) == false)
	{
		std::cerr << "parse html fail!" << std::endl;
		return 2;
	}

	// 3.把解析完毕的各个文件内容写入到output中，按照 \3 作为每个文档的分隔符
	if (SaveHtml(results, output) == false)
	{
		std::cerr << "save html fail!" << std::endl;
		return 3;
	}
	return 0;
}
